package cn.earas.htmlX.service;

import cn.earas.htmlX.model.UrlModel;
import cn.earas.htmlX.net.HttpUtil;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.*;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Core {

    private Set<String> usedUrls = Collections.synchronizedSet(new HashSet<String>());

    private String charset;

    private int connectTimeout;

    private int socketTimeout;

    private int connectionRequestTimeout;

    private int waitJavaScriptTime;

    private String dir;

    private String baseHost;

    private AtomicInteger threads;

    private boolean ignoreOutUrl;

    private boolean doJs;

    private String cookie;

    private String userAgent;

    private int lvl;

    private String cssPath;

    private String jsPath;

    private String imgPath;

    private String fontPath;

    private String otherPath;

    private HttpHost httpHost;

    public Core() {
        dir = null;
        ignoreOutUrl = true;
        doJs = false;
        cookie = null;
        userAgent = null;
        lvl = 0;
        cssPath = "css";
        jsPath = "js";
        imgPath = "images";
        fontPath = "fonts";
        otherPath = "others";
        waitJavaScriptTime = 5000;
        charset = "utf-8";
        connectTimeout = 10000;
        connectionRequestTimeout = 10000;
        socketTimeout = 10000;
        httpHost = null;
    }

    public void enableDoJs() {
        doJs = true;
        this.waitJavaScriptTime = this.connectTimeout;
    }

    public void enableAllUrl(int lvl) {
        this.lvl = lvl;
        threads = new AtomicInteger(0);
    }

    public void enableOutUrl(String domain) {
        this.ignoreOutUrl = false;
        this.baseHost = domain.replace("http://", "").replace("https://", "");
    }

    public void enableProxy(String host, int port, String proxyType) {
        httpHost = new HttpHost(host, port, proxyType);
    }

    public void setCharset(String charset) {
        this.charset = charset;
    }

    public void setOutTime(int outTime) {
        this.connectionRequestTimeout = outTime;
        this.connectTimeout = outTime;
        this.socketTimeout = outTime;
    }

    public void setCookie(String cookie) {
        this.cookie = cookie;
    }

    public void setUserAgent(String userAgent) {
        this.userAgent = userAgent;
    }

    public String setPaths(String cssPath, String jsPath, String imgPath, String fontPath, String otherPath) {
        if (checkPath(cssPath)) this.cssPath = cssPath;
        else return "css路径错误";
        if (checkPath(jsPath)) this.jsPath = jsPath;
        else return "js路径错误";
        if (checkPath(imgPath)) this.imgPath = imgPath;
        else return "images路径错误";
        if (checkPath(fontPath)) this.fontPath = fontPath;
        else return "fonts路径错误";
        if (checkPath(otherPath)) this.otherPath = otherPath;
        else return "other路径错误";
        return "ok";
    }

    public String execute(List<UrlModel> urls, String path) {
        try {
            //创建文件夹
            File rootDir = mkdir(urls.get(0).getUrl(), path);
            for (UrlModel urlModel : urls) {
                try {
                    //开始执行
                    urlToFile(urlModel.getUrl(), urlModel.getName(), lvl);
                } catch (Exception e) {
                    e.printStackTrace();
                    System.out.println("运行出错，链接：" + urlModel.getUrl() + "，文件名：" + urlModel.getName() +"，错误信息："+ e.getMessage());
                }
            }
            if (lvl > 0) {
                while (threads.intValue() > 0) {
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
            //删除空目录
            deleteEmptyDir(rootDir);
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println(e.getMessage());
        }
        //清空
        usedUrls.clear();
        //返回生成路径
        return dir;
    }

    public String execute(UrlModel urlModel, String path) {
        try {
            //创建文件夹
            File rootDir = mkdir(urlModel.getUrl(), path);
            //开始执行
            urlToFile(urlModel.getUrl(), urlModel.getName(), lvl);
            if (lvl > 0) {
                while (threads.intValue() > 0) {
                    System.out.println(threads);
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                        System.out.println(e.getMessage());
                    }
                }
            }
            //删除空目录
            deleteEmptyDir(rootDir);
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("运行出错，链接：" + urlModel.getUrl() + "，文件名：" + urlModel.getName() +"，错误信息："+ e.getMessage());
        }
        //清空
        usedUrls.clear();
        //返回生成路径
        return dir;
    }

    private File mkdir(String url, String path) {
        String temp = url.replace("http://", "").replace("https://", "");
        if (!temp.endsWith("/")) temp += "/";
        dir = path + File.separator + temp.substring(0, temp.indexOf("/"));
        File rootDir = new File(dir);
        if (!rootDir.exists()) {
            rootDir.mkdirs();
            File cssDir = new File(rootDir.getPath() + File.separator + cssPath);
            if (!cssDir.exists()) cssDir.mkdirs();
            File jsDir = new File(rootDir.getPath() + File.separator + jsPath);
            if (!jsDir.exists()) jsDir.mkdirs();
            File imgDir = new File(rootDir.getPath() + File.separator + imgPath);
            if (!imgDir.exists()) imgDir.mkdirs();
            File fontDir = new File(rootDir.getPath() + File.separator + fontPath);
            if (!fontDir.exists()) fontDir.mkdirs();
            File otherDir = new File(rootDir.getPath() + File.separator + otherPath);
            if (!otherDir.exists()) otherDir.mkdirs();
        }
        return rootDir;
    }

    private void deleteEmptyDir(File file) {
        for (File listFile : file.listFiles()) {
            if (listFile.isDirectory()) deleteEmptyDir(listFile);
        }
        if (file.listFiles().length == 0) file.delete();
    }

    private void urlToFile(String url, String name, int lvl) {
        if (!url.endsWith("/")) url += "/";
        String rootUrl = url.replace("://", "!@#");
        rootUrl = rootUrl.substring(0, rootUrl.indexOf("/")).replace("!@#", "://");
        //获取client
        HttpUtil httpUtil = new HttpUtil();
        RequestConfig requestConfig = RequestConfig.custom()
                .setConnectTimeout(connectTimeout)// 一、连接超时：connectionTimeout-->指的是连接一个url的连接等待时间
                .setSocketTimeout(socketTimeout)// 二、读取数据超时：SocketTimeout-->指的是连接上一个url，获取response的返回等待时间
                .setConnectionRequestTimeout(connectionRequestTimeout)
                .setProxy(httpHost)
                .build();
        if (usedUrls.contains(url)) return;
        usedUrls.add(url);
        String html = httpUtil.getHtml(doGet(url, requestConfig), charset);
        if (html.equals("")) {
            httpUtil.close();
            return;
        }
        Document document = Jsoup.parse(html);
        Elements metas = document.getElementsByTag("meta");
        for (Element meta : metas) {
            if (meta.hasAttr("charset")) {
                String tempCharset = meta.attr("charset");
                if (!tempCharset.toLowerCase().equals(charset)) {
                    charset = tempCharset;
                    html = httpUtil.getHtml(doGet(url, requestConfig), charset);
                    document = Jsoup.parse(html);
                }
                break;
            }
        }
        if (doJs) {
            try {
                String tempHtml = doJs(url);
                document = Jsoup.parse(tempHtml);
            } catch (IOException e) {
                e.printStackTrace();
                System.out.println("执行js出错，错误信息："+e.getMessage());
            }
        }
        //img
        Elements imgs = document.getElementsByTag("img");
        for (Element img : imgs) {
            try {
                //获取文件真实url
                String href = getUrl(rootUrl, img.attr("src"));
                if (href.equals("")) continue;
                //获取文件名
                String fileName = getFileName(href);
                if (fileName.equals("")) continue;
                File imgFile = new File(dir + File.separator + imgPath + File.separator + fileName);
                img.attr("src", imgPath + "/" + fileName);
                if (img.hasAttr("srcset")) {
                    String[] srcsets = img.attr("srcset").split(",");
                    String srcset = "";
                    for (String s : srcsets) {
                        String suffix = s.substring(s.lastIndexOf(" "));
                        String src = getUrl(rootUrl, s.substring(0, s.lastIndexOf(" ")));
                        if (src.equals("")) continue;
                        //获取文件名
                        String fileName2 = getFileName(src);
                        if (fileName2.equals("")) continue;
                        File imgFile2 = new File(dir + File.separator + imgPath + File.separator + fileName2);
                        srcset = srcset + imgPath + "/" + fileName + suffix + ",";
                        if (usedUrls.contains(src)) continue;
                        usedUrls.add(src);
                        File temp2 = httpUtil.download(doGet(src, requestConfig));
                        fileToFile(temp2, imgFile2);
                    }
                    if (srcset.endsWith(",")) srcset = srcset.substring(0, srcset.length() - 1);
                    img.attr("srcset", srcset);
                }
                if (usedUrls.contains(href)) continue;
                usedUrls.add(href);
                File temp = httpUtil.download(doGet(href, requestConfig));
                fileToFile(temp, imgFile);
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println("获取文件失败，错误信息："+e.getMessage());
            }
        }
        //scripts
        Elements scripts = document.getElementsByTag("script");
        for (Element script : scripts) {
            try {
                if (script.hasAttr("src")) {
                    //获取文件真实url
                    String href = getUrl(rootUrl, script.attr("src"));
                    if (href.equals("")) continue;
                    //获取文件名
                    String fileName = getFileName(href);
                    if (fileName.equals("")) continue;
                    if (!fileName.endsWith(".js")) fileName = fileName + ".js";
                    File jsFile = new File(dir + File.separator + jsPath + File.separator + fileName);
                    script.attr("src", jsPath + "/" + fileName);
                    if (usedUrls.contains(href)) continue;
                    usedUrls.add(href);
                    File temp = httpUtil.download(doGet(href, requestConfig));
                    fileToFile(temp, jsFile);
                }
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println("获取文件失败，错误信息："+e.getMessage());
            }
        }
        //links
        Elements links = document.getElementsByTag("link");
        for (Element link : links) {
            try {
                //获取文件真实url
                String href = getUrl(rootUrl, link.attr("href"));
                if (href.equals("")) continue;
                //获取文件名
                String fileName = getFileName(href);
                if (fileName.equals("")) continue;
                //将文件写到对应目录
                if (link.attr("type").toLowerCase().equals("text/css") || link.attr("rel").toLowerCase().equals("stylesheet")) {
                    link.attr("href", cssPath + "/" + fileName);
                    if (usedUrls.contains(href)) continue;
                    usedUrls.add(href);
                    String cssText = httpUtil.getHtml(doGet(href, requestConfig), charset);
                    File cssFile = new File(dir + File.separator + cssPath + File.separator + fileName);
                    stringToFile(href, cssText, cssFile, false);
                } else if (link.attr("type").toLowerCase().contains("image/") || link.attr("rel").toLowerCase().contains("icon")) {
                    File imgFile = new File(dir + File.separator + imgPath + File.separator + fileName);
                    link.attr("href", imgPath + "/" + fileName);
                    if (usedUrls.contains(href)) continue;
                    usedUrls.add(href);
                    File temp = httpUtil.download(doGet(href, requestConfig));
                    fileToFile(temp, imgFile);
                } else if (link.hasAttr("as") && link.attr("as").equals("script")) {
                    if (!fileName.endsWith(".js")) fileName = fileName + ".js";
                    File jsFile = new File(dir + File.separator + jsPath + File.separator + fileName);
                    link.attr("href", jsPath + "/" + fileName);
                    if (usedUrls.contains(href)) continue;
                    usedUrls.add(href);
                    File temp = httpUtil.download(doGet(href, requestConfig));
                    fileToFile(temp, jsFile);
                } else {
                    if (link.attr("rel").equals("canonical")) continue;
                    File otherFile = new File(dir + File.separator + otherPath + File.separator + fileName);
                    link.attr("href", otherPath + "/" + fileName);
                    if (usedUrls.contains(href)) continue;
                    usedUrls.add(href);
                    File temp = httpUtil.download(doGet(href, requestConfig));
                    fileToFile(temp, otherFile);
                }
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println("获取文件失败，错误信息："+e.getMessage());
            }
        }
        if (lvl > 0) {
            getUrls(document, rootUrl, --lvl);
        }
        File htmlFile = new File(dir + File.separator + name);
        String text = document.html().toLowerCase().startsWith("<!doctype html>") ? document.html() : "<!DOCTYPE html>" + document.html();
        stringToFile(url, text, htmlFile, true);
        //释放client
        httpUtil.close();
    }

    private HttpGet doGet(String url, RequestConfig requestConfig) {
        HttpGet get = new HttpGet(url);
        if (userAgent == null)
            get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36");
        else get.setHeader("User-Agent", userAgent);
        get.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        get.setHeader("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
        get.setHeader("Accept-Encoding", "gzip, deflate");
        if (cookie != null) get.setHeader("Cookie", cookie);
        get.setConfig(requestConfig);
        return get;
    }

    private void fileToFile(File temp, File file) {
        if (temp == null) return;
        try {
            if (!file.exists()) FileUtils.copyFile(temp, file);
        } catch (IOException e) {
            e.printStackTrace();
            System.out.println("生成文件失败，错误信息："+e.getMessage());
        }
        if (temp.exists()) {
            System.gc();
            temp.delete();
        }
    }

    private void stringToFile(String parentUrl, String text, File file, boolean isPage) {
        try {
            String tempStr = text;
            HttpUtil httpUtil = new HttpUtil();
            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(connectTimeout)// 一、连接超时：connectionTimeout-->指的是连接一个url的连接等待时间
                    .setSocketTimeout(socketTimeout)// 二、读取数据超时：SocketTimeout-->指的是连接上一个url，获取response的返回等待时间
                    .setConnectionRequestTimeout(connectionRequestTimeout)
                    .setProxy(httpHost)
                    .build();
            if (!file.getParentFile().exists()) file.getParentFile().mkdirs();
            file.createNewFile();
            Pattern pattern = Pattern.compile("url\\([^\\s^;^,^{*]+\\)");
            Matcher matcher = pattern.matcher(tempStr);
            while (matcher.find()) {
                try {
                    String rootUrl = parentUrl;
                    if (!rootUrl.endsWith("/")) rootUrl += "/";
                    rootUrl = rootUrl.replace("://", "!@#");
                    rootUrl = rootUrl.substring(0, rootUrl.indexOf("/")).replace("!@#", "://");
                    String tempParentUrl = parentUrl;
                    String str = matcher.group();
                    str = str.replace("url(", "").replace(")", "").replace("'", "").replace("\"", "");
                    String url = str;
                    if (!url.contains("http")) {
                        if (url.replace(" ", "").equals("")) continue;
                        if (url.startsWith("//")) url = "http:" + url;
                        else {
                            if (url.startsWith("/")) {
                                url = rootUrl + url;
                            } else {
                                tempParentUrl = tempParentUrl.substring(0, tempParentUrl.lastIndexOf("/"));
                                while (url.startsWith("../")) {
                                    url = url.substring(3);
                                    tempParentUrl = tempParentUrl.substring(0, tempParentUrl.lastIndexOf("/"));
                                }
                                url = tempParentUrl + "/" + url;
                            }
                        }
                    }
                    String fileName = getFileName(str);
                    if (fileName.equals("")) continue;
                    String type = otherPath;
                    if (fileName.endsWith(".png")
                            || fileName.endsWith(".bmp")
                            || fileName.endsWith(".gif")
                            || fileName.endsWith(".jpeg")
                            || fileName.endsWith(".ico")
                            || fileName.endsWith(".jpe")
                            || fileName.endsWith(".pic")
                            || fileName.endsWith(".jpg")) {
                        type = imgPath;
                    } else if (fileName.endsWith(".ttf")
                            || fileName.endsWith(".woff")
                            || fileName.endsWith(".eot")
                            || fileName.endsWith(".otf")
                            || fileName.endsWith(".woff2")
                            || fileName.endsWith(".svg")) {
                        type = fontPath;
                    }
                    File imgFile = new File(dir + File.separator + type + File.separator + fileName);
                    if (usedUrls.contains(url)) continue;
                    usedUrls.add(url);
                    File temp = httpUtil.download(doGet(url, requestConfig));
                    fileToFile(temp, imgFile);
                    if (isPage) text = text.replace(str, type + "/" + fileName);
                    else text = text.replace(str, "../" + type + "/" + fileName);
                } catch (Exception e) {
                    e.printStackTrace();
                    System.out.println("获取文件失败，错误信息："+e.getMessage());
                }
            }
            FileWriter fileWriter = new FileWriter(file);
            fileWriter.write(text);
            fileWriter.flush();
            fileWriter.close();
        } catch (IOException e) {
            e.printStackTrace();
            System.out.println("获取文件失败，错误信息："+e.getMessage());
        }
    }

    private String doJs(String url) throws IOException {
        WebClient webClient = new WebClient();
        // 1 启动JS
        webClient.getOptions().setJavaScriptEnabled(true);
        // 2 禁用Css，可避免自动二次请求CSS进行渲染
        webClient.getOptions().setCssEnabled(false);
        // 3 启动客户端重定向
        webClient.getOptions().setRedirectEnabled(true);
        // 4 js运行错误时，是否抛出异常
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        // 5 设置超时
        webClient.getOptions().setTimeout(connectTimeout);
        HtmlPage htmlPage = webClient.getPage(url);
        // 等待JS驱动dom完成获得还原后的网页
        webClient.waitForBackgroundJavaScript(waitJavaScriptTime);
        webClient.close();
        return htmlPage.asXml();
    }

    private void getUrls(Document document, String rootUrl, int lvl) {
        Elements as = document.getElementsByTag("a");
        for (Element a : as) {
            try {
                if (!a.hasAttr("href")) continue;
                String href = a.attr("href");
                if (href.replace("#", "").equals("") || href.replace(" ", "").equals("")) continue;
                if (href.contains("javascript:")) continue;

                if (href.startsWith("mailto:")) continue;
                if (!href.contains("//")) {
                    if (href.startsWith("/")) href = rootUrl + href;
                    else href = rootUrl + "/" + href;
                } else {
                    if (!href.startsWith("htt")) href = "http:" + href;
                }
                if (ignoreOutUrl && !href.contains(baseHost)) continue;
                String name = href.replace(rootUrl + "/", "");
                if (name.contains("?")) name = name.substring(0, name.indexOf("?"));
                if (name.endsWith("/")) name = name.substring(0, name.length() - 1);
                if (name.replace("/", "").equals("")) continue;
                if (name.contains("/")) name = name.substring(name.lastIndexOf("/") + 1);
                if (name.contains(".")) name = name.substring(0, name.indexOf("."));
                if (name.contains("#")) continue;
                new Thread(new ExecuteThread(href, name + ".html", lvl)).start();
                a.attr("href", name + ".html");
            } catch (Exception e) {
                e.printStackTrace();
                System.out.println("获取文件失败，错误信息："+e.getMessage());
            }
        }
    }

    private boolean checkPath(String path) {
        try {
            if (path != null && !path.replace(" ", "").equals("")) {
                if (path.startsWith("/")) path = path.substring(1);
                if (path.endsWith("/")) path = path.substring(0, path.length() - 1);
            } else return false;
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("路径错误，错误信息："+e.getMessage());
            return false;
        }
        return true;
    }

    private String getFileName(String url) {
        String fileName = url.substring(url.lastIndexOf("/") + 1);
        if (fileName.contains("?")) fileName = fileName.substring(0, fileName.indexOf("?"));
        if (fileName.contains("#")) fileName = fileName.substring(0, fileName.indexOf("#"));
        if (fileName.replace(" ", "").equals("")) fileName = "";
        return fileName;
    }

    private String getUrl(String rootUrl, String href) {
        if (!href.contains(".")) href = "";
        if (!href.contains("http://") && !href.contains("https://")) {
            if (href.startsWith("//")) {
                href = "http:" + href;
            } else {
                if (href.startsWith("/")) href = rootUrl + href;
                else href = rootUrl + "/" + href;
            }
        }
        return href;
    }

    class ExecuteThread implements Runnable {

        private String name;

        private String url;

        private int lvl;

        public ExecuteThread(String url, String name, int lvl) {
            this.url = url;
            this.name = name;
            this.lvl = lvl;
        }

        public void run() {
            threads.incrementAndGet();
            while (threads.intValue() > 500) {
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                    System.out.println(e.getMessage());
                }
            }
            urlToFile(url, name, lvl);
            threads.decrementAndGet();
        }
    }
}
